/**
 *  fetch by crawler
 * @param shops
 */


const Crawler = require("crawler");



var c = new Crawler({
  jQuery: true,
  maxConnections: 10,
  // This will be called for each crawled page
  callback: function (error, res, done) {
    if (error) {
      console.log(error);
    } else {
      var $ = res.$;
      console.log('c')
    }
    done();
  }
}); 
module.exports = (shops) => {
  if (shops && shops.length > 0){
    let shop = undefined;
    while (shop = shops.shift()) {
      c.queue([{
        uri: `${shop.shopUrl}`,
        jQuery: true,
        rateLimit: 1000,
        retries:5,
        // The global callback won't be called
        callback: function (error, res, done) {
          if (error) {
            console.log(error);
          } else {
            var $ = res.$;
          }
          done();
        }
      }]);
    }
    

  }  
}